{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 处理重复数据",
   "id": "7c9cfe508686a1ac"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:15:55.494947Z",
     "start_time": "2025-01-08T16:15:54.971500Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ],
   "id": "b6749a6b792be4fb",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:16:58.344894Z",
     "start_time": "2025-01-08T16:16:58.336891Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df_obj=pd.DataFrame({'data1' : ['a'] * 4 + ['b'] * 4,\n",
    "                       'data2' : np.random.randint(0, 4, 8)})\n",
    "\n",
    "print(df_obj)"
   ],
   "id": "3081ba2e34b1c808",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  data1  data2\n",
      "0     a      3\n",
      "1     a      0\n",
      "2     a      0\n",
      "3     a      2\n",
      "4     b      3\n",
      "5     b      1\n",
      "6     b      3\n",
      "7     b      3\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:17:31.094876Z",
     "start_time": "2025-01-08T16:17:31.089103Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# duplicates()函数可以查看重复数据\n",
    "# 返回布尔值Series，True表示重复数据，False表示不重复数据\n",
    "# 默认比较行向量中的所有元素，如果要比较列，需要指定axis=1\n",
    "print(df_obj.duplicated())"
   ],
   "id": "a1e4e38f762cadc0",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    False\n",
      "1    False\n",
      "2     True\n",
      "3    False\n",
      "4    False\n",
      "5    False\n",
      "6     True\n",
      "7     True\n",
      "dtype: bool\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:21:19.462893Z",
     "start_time": "2025-01-08T16:21:19.452323Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 取出不重复行\n",
    "df_obj[~df_obj.duplicated()]"
   ],
   "id": "4c2119c6857b66ea",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  data1  data2\n",
       "0     a      3\n",
       "1     a      0\n",
       "3     a      2\n",
       "4     b      3\n",
       "5     b      1"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>b</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:23:53.074291Z",
     "start_time": "2025-01-08T16:23:53.069585Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# duplicates()函数可以指定比较向量中的哪些元素，默认比较所有元素\n",
    "print(df_obj.duplicated('data2'))"
   ],
   "id": "9ce26d51ddd5e5e8",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    False\n",
      "1    False\n",
      "2     True\n",
      "3    False\n",
      "4     True\n",
      "5    False\n",
      "6     True\n",
      "7     True\n",
      "dtype: bool\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:25:55.919175Z",
     "start_time": "2025-01-08T16:25:55.911224Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df_obj1=pd.DataFrame({'data1' :[np.nan] * 4,\n",
    "                       'data2' :list('1235')})\n",
    "df_obj1"
   ],
   "id": "9df25f871783d1b9",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "   data1 data2\n",
       "0    NaN     1\n",
       "1    NaN     2\n",
       "2    NaN     3\n",
       "3    NaN     5"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:26:15.148752Z",
     "start_time": "2025-01-08T16:26:15.142481Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#在pd的duplicated认为空值和空值相等的\n",
    "df_obj1.duplicated('data1')"
   ],
   "id": "5eab1b2353039a0a",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1     True\n",
       "2     True\n",
       "3     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:27:49.965821Z",
     "start_time": "2025-01-08T16:27:49.961052Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# drop_duplicates()函数可以删除重复行\n",
    "# 默认删除所有重复行，如果指定subset，则只删除指定列的重复行\n",
    "print(df_obj.drop_duplicates(\"data2\"))  "
   ],
   "id": "a20f7d5976d394af",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  data1  data2\n",
      "0     a      3\n",
      "1     a      0\n",
      "3     a      2\n",
      "5     b      1\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:27:52.077105Z",
     "start_time": "2025-01-08T16:27:52.070988Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#如果要在原有的df上去重，需要加inplace=True\n",
    "df_obj"
   ],
   "id": "c0387b57b86a283",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  data1  data2\n",
       "0     a      3\n",
       "1     a      0\n",
       "2     a      0\n",
       "3     a      2\n",
       "4     b      3\n",
       "5     b      1\n",
       "6     b      3\n",
       "7     b      3"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>data2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>b</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>b</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>b</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:28:59.848383Z",
     "start_time": "2025-01-08T16:28:59.842563Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#map与applymap一样，但是map只能用于series，applymap只能用于df\n",
    "ser_obj = pd.Series(np.random.randint(0,10,10))  #series 用map\n",
    "print(ser_obj)\n",
    "print(\"-\"*50)\n",
    "print(ser_obj.map(lambda x : x ** 2))"
   ],
   "id": "84bf3c41653b008b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    5\n",
      "1    1\n",
      "2    5\n",
      "3    9\n",
      "4    2\n",
      "5    1\n",
      "6    3\n",
      "7    6\n",
      "8    0\n",
      "9    3\n",
      "dtype: int32\n",
      "--------------------------------------------------\n",
      "0    25\n",
      "1     1\n",
      "2    25\n",
      "3    81\n",
      "4     4\n",
      "5     1\n",
      "6     9\n",
      "7    36\n",
      "8     0\n",
      "9     9\n",
      "dtype: int64\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:32:38.943407Z",
     "start_time": "2025-01-08T16:32:38.938151Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#异常值手动替换\n",
    "ser_obj=pd.Series(np.arange(10),index=range(3,13))\n",
    "print(ser_obj)\n",
    "print('-' * 20)\n",
    "\n",
    "# 单个值替换单个值\n",
    "print(ser_obj.replace(1, -100))\n",
    "print('-' * 20)\n",
    "\n",
    "# 多个值替换一个值\n",
    "print(ser_obj.replace(range(6,9), -100))\n",
    "print('-' * 20)\n",
    "\n",
    "# 多个值替换多个值\n",
    "print(ser_obj.replace([4, 7], [-100, -200]))"
   ],
   "id": "ae24e6ffa96bb776",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3     0\n",
      "4     1\n",
      "5     2\n",
      "6     3\n",
      "7     4\n",
      "8     5\n",
      "9     6\n",
      "10    7\n",
      "11    8\n",
      "12    9\n",
      "dtype: int64\n",
      "--------------------\n",
      "3       0\n",
      "4    -100\n",
      "5       2\n",
      "6       3\n",
      "7       4\n",
      "8       5\n",
      "9       6\n",
      "10      7\n",
      "11      8\n",
      "12      9\n",
      "dtype: int64\n",
      "--------------------\n",
      "3       0\n",
      "4       1\n",
      "5       2\n",
      "6       3\n",
      "7       4\n",
      "8       5\n",
      "9    -100\n",
      "10   -100\n",
      "11   -100\n",
      "12      9\n",
      "dtype: int64\n",
      "--------------------\n",
      "3       0\n",
      "4       1\n",
      "5       2\n",
      "6       3\n",
      "7    -100\n",
      "8       5\n",
      "9       6\n",
      "10   -200\n",
      "11      8\n",
      "12      9\n",
      "dtype: int64\n"
     ]
    }
   ],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:33:11.700277Z",
     "start_time": "2025-01-08T16:33:11.692829Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df = pd.DataFrame({'A': [0, 1, 2, 3, 4],\n",
    "                   'B': [5, 6, 7, 8, 9],\n",
    "                   'C': ['a', 'b', 'ac', 'd', 'e']})\n",
    "\n",
    "df"
   ],
   "id": "baaf34817397806b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "   A  B   C\n",
       "0  0  5   a\n",
       "1  1  6   b\n",
       "2  2  7  ac\n",
       "3  3  8   d\n",
       "4  4  9   e"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>ac</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>e</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:33:52.422013Z",
     "start_time": "2025-01-08T16:33:52.414730Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#正则表达式替换\n",
    "df.replace(to_replace=r'^a', value=100, regex=True)"
   ],
   "id": "beb738f5143d1030",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "   A  B    C\n",
       "0  0  5  100\n",
       "1  1  6    b\n",
       "2  2  7  100\n",
       "3  3  8    d\n",
       "4  4  9    e"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>e</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-08T16:34:08.232923Z",
     "start_time": "2025-01-08T16:34:08.227322Z"
    }
   },
   "cell_type": "code",
   "source": "df.dtypes",
   "id": "b15dd4442bd8f14b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "A     int64\n",
       "B     int64\n",
       "C    object\n",
       "dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
